//package com.zxl.grocery.webmagic.processor;
//
//import com.baomidou.mybatisplus.toolkit.StringUtils;
//import com.zxl.grocery.core.support.StrKit;
//import com.zxl.grocery.core.util.ListUtils;
//import com.zxl.grocery.core.util.SpringContextHolder;
//import com.zxl.grocery.core.util.ToolUtil;
//import com.zxl.grocery.webmagic.common.persistence.model.CrawlerNovel;
//import com.zxl.grocery.webmagic.dao.NovelMgrDao;
//import us.codecraft.webmagic.Page;
//import us.codecraft.webmagic.Site;
//import us.codecraft.webmagic.processor.PageProcessor;
//
//import javax.annotation.Resource;
//import java.util.List;
//
//
///**
// * info:笔趣阁爬虫程序
// * Created by zxl on 2017/8/8 0008.
// */
//public class BiQuGeProcessor1 implements PageProcessor {
//
//    public static final String shengXuUrl = "http://www.biqiuge.com/book/4772/";//圣墟列表页
//    public static final String SX = "http://www.biqiuge.com/book/4772/\\d+.html";//圣墟详细页
//
//    public static final String yiNianYongHengUrl = "http://www.biqiuge.com/book/630/";//一念永恒列表页
//    public static final String YNYH = "http://www.biqiuge.com/book/630/\\d+.html";//一念永恒详细页
//
//    public static final String woShiZhiZunUrl = "http://www.biqiuge.com/book/2956/";//我是至尊列表页
//    public static final String WSZZ = "http://www.biqiuge.com/book/2956/\\d+.html";//我是至尊详细页
//
//    @Resource
//    private NovelMgrDao novelMgrDao = SpringContextHolder.getBean(NovelMgrDao.class);
//    // 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
//    private Site site = Site.me()
//            .setDomain("http://www.biqiuge.com")
//            .setSleepTime(3000)
//            .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
//
//    @Override
//    // process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
//    public void process(Page page) {
//        System.out.println(page.getUrl());
//        if(page.getUrl().regex(shengXuUrl).match()){
//            String sxAuthor = null;
//            String sxTitle = null;
//            if (!page.getUrl().regex(SX).match()) {
//                sxAuthor = page.getHtml().xpath("//*[@id=\"info\"]/p[1]/a/text()").toString();
//                sxTitle = page.getHtml().xpath("//*[@id=\"info\"]/h1/text()").toString();
////                System.out.println("title:"+ title + "author:"+author);
//                List<String> sxUrls = page.getHtml().xpath("//div[@id='list']").links().regex(SX).all();
//                List<String> sxExistLinks = novelMgrDao.getAllLinksByTypeId(1);
//                List<String> sxNeedAddUrls = ListUtils.getDiffrentLists(sxUrls,sxExistLinks);
//                System.out.println("圣墟新增的urls:"+sxNeedAddUrls.size());
//                page.addTargetRequests(sxNeedAddUrls);
//            } else {
//                String chapter=page.getHtml().xpath("//div[@class='bookname']/h1/text()").toString();
//                String content=page.getHtml().xpath("//div[@id='content']/text()").toString();
//                String link = page.getUrl().toString();
//                Integer chapter_sort = ToolUtil.getChapterSort(chapter);
//                System.out.println("chapter:::"+chapter);
//                System.out.println(content);
//                if(content != null){
//                    if(StrKit.str1ContainsStr2(content,"笔趣阁 www.biqiuge.com，最快更新圣墟最新章节！")){
//                        content = (content.replaceAll("笔趣阁 www.biqiuge.com，最快更新圣墟最新章节！",""));
//                        System.out.println(content);
//                    }
//                }
//                if (StringUtils.isNotEmpty(content)) {
//                    CrawlerNovel novel = new CrawlerNovel();
//                    novel.setTitle(sxTitle);
//                    novel.setNovelist(sxAuthor);
//                    novel.setChapterSort(chapter_sort);
//                    novel.setChapter(chapter);
//                    novel.setContent(content);
//                    novel.setLink(link);
//                    novel.setTypeId(1);
//                    novel.setSourcesId(8);
//                    page.putField("novel", novel);
//                }
//            }
//        }
//        System.out.println(page.getUrl());
//        if(page.getUrl().regex(yiNianYongHengUrl).match()){
//            String ynyhAuthor = null;
//            String ynyhTitle = null;
//            if(!page.getUrl().regex(YNYH).match()){
//                ynyhAuthor = page.getHtml().xpath("//*[@id=\"info\"]/p[1]/a/text()").toString();
//                ynyhTitle = page.getHtml().xpath("//*[@id=\"info\"]/h1/text()").toString();
//                List<String> ynyhUrls = page.getHtml().xpath("//div[@id='list']").links().regex(YNYH).all();
//                List<String> ynyhExistLinks = novelMgrDao.getAllLinksByTypeId(3);
//                List<String> ynyhNeedAddUrls = ListUtils.getDiffrentLists(ynyhUrls,ynyhExistLinks);
//                System.out.println("一念永恒新增的urls:"+ynyhNeedAddUrls.size());
//                page.addTargetRequests(ynyhNeedAddUrls);
//            } else {
//                String chapter=page.getHtml().xpath("//div[@class='bookname']/h1/text()").toString();
//                String content=page.getHtml().xpath("//div[@id='content']/text()").toString();
//                String link = page.getUrl().toString();
//                Integer chapter_sort = ToolUtil.getChapterSort(chapter);
//                System.out.println("chapter:::"+chapter);
//                System.out.println(content);
//                if(content != null){
//                    if(StrKit.str1ContainsStr2(content,"笔趣阁 www.biqiuge.com，最快更新一念永恒最新章节！")){
//                        content = (content.replaceAll("笔趣阁 www.biqiuge.com，最快更新一念永恒最新章节！",""));
//                        System.out.println(content);
//                    }
//                }
//                if (StringUtils.isNotEmpty(content)) {
//                    CrawlerNovel novel = new CrawlerNovel();
//                    novel.setTitle(ynyhTitle);
//                    novel.setNovelist(ynyhAuthor);
//                    novel.setChapterSort(chapter_sort);
//                    novel.setChapter(chapter);
//                    novel.setContent(content);
//                    novel.setLink(link);
//                    novel.setTypeId(3);
//                    novel.setSourcesId(8);
//                    page.putField("novel", novel);
//                }
//            }
//        }
//        System.out.println(page.getUrl());
//        if(page.getUrl().regex(woShiZhiZunUrl).match()){
//            String wszzAuthor = null;
//            String wszzTitle = null;
//            if(!page.getUrl().regex(WSZZ).match()){
//                wszzAuthor = page.getHtml().xpath("//*[@id=\"info\"]/p[1]/a/text()").toString();
//                wszzTitle = page.getHtml().xpath("//*[@id=\"info\"]/h1/text()").toString();
//                List<String> ynyhUrls = page.getHtml().xpath("//div[@id='list']").links().regex(WSZZ).all();
//                List<String> ynyhExistLinks = novelMgrDao.getAllLinksByTypeId(4);
//                List<String> ynyhNeedAddUrls = ListUtils.getDiffrentLists(ynyhUrls,ynyhExistLinks);
//                System.out.println("我是至尊新增的urls:"+ynyhNeedAddUrls.size());
//                page.addTargetRequests(ynyhNeedAddUrls);
//            } else {
//                String chapter=page.getHtml().xpath("//div[@class='bookname']/h1/text()").toString();
//                String content=page.getHtml().xpath("//div[@id='content']/text()").toString();
//                String link = page.getUrl().toString();
//                Integer chapter_sort = ToolUtil.getChapterSort(chapter);
//                System.out.println("chapter:::"+chapter);
//                System.out.println(content);
//                if(content != null){
//                    if(StrKit.str1ContainsStr2(content,"笔趣阁 www.biqiuge.com，最快更新我是至尊最新章节！")){
//                        content = (content.replaceAll("笔趣阁 www.biqiuge.com，最快更新我是至尊最新章节！",""));
//                        System.out.println(content);
//                    }
//                }
//                if (StringUtils.isNotEmpty(content)) {
//                    CrawlerNovel novel = new CrawlerNovel();
//                    novel.setTitle(wszzTitle);
//                    novel.setNovelist(wszzAuthor);
//                    novel.setChapterSort(chapter_sort);
//                    novel.setChapter(chapter);
//                    novel.setContent(content);
//                    novel.setLink(link);
//                    novel.setTypeId(4);
//                    novel.setSourcesId(8);
//                    page.putField("novel", novel);
//                }
//            }
//        }
//        /*if (!page.getUrl().regex(SX).match()) {
//            List<String> sxUrls = page.getHtml().xpath("//div[@id='list']").links().regex(SX).all();
//            List<String> sxExistLinks = novelMgrDao.getAllLinksByTypeId(1);
//            List<String> sxNeedAddUrls = ListUtils.getDiffrentLists(sxUrls,sxExistLinks);
//            System.out.println("圣墟新增的urls:"+sxNeedAddUrls.size());
//            page.addTargetRequests(sxNeedAddUrls);
//        }else if(!page.getUrl().regex(YNYH).match()){
//            List<String> ynyhUrls = page.getHtml().xpath("//div[@id='list']").links().regex(YNYH).all();
//            List<String> ynyhExistLinks = novelMgrDao.getAllLinksByTypeId(3);
//            List<String> ynyhNeedAddUrls = ListUtils.getDiffrentLists(ynyhUrls,ynyhExistLinks);
//            System.out.println("一念永恒新增的urls:"+ynyhNeedAddUrls.size());
//            page.addTargetRequests(ynyhNeedAddUrls);
//        } else {
//            String chapter=page.getHtml().xpath("//div[@class='bookname']/h1/text()").toString();
//            String content=page.getHtml().xpath("//div[@id='content']/text()").toString();
//            String link = page.getUrl().toString();
//            System.out.println("chapter:::"+chapter);
//            System.out.println(content);
//            if(content != null){
//                if(StrKit.str1ContainsStr2(content,"笔趣阁 www.biqiuge.com，最快更新圣墟最新章节！")){
//                    content = (content.replaceAll("笔趣阁 www.biqiuge.com，最快更新圣墟最新章节！",""));
//                    if (StringUtils.isNotEmpty(content)) {
//                        CrawlerNovel novel = new CrawlerNovel();
//                        novel.setTitle("圣墟");
//                        novel.setChapter(chapter);
//                        novel.setContent(content);
//                        novel.setLink(link);
//                        novel.setTypeId(1);
//                        novel.setSourcesId(8);
//                        page.putField("novel", novel);
//                    }
//                }else if(StrKit.str1ContainsStr2(content,"笔趣阁 www.biqiuge.com，最快更新一念永恒最新章节！")){
//                    content = (content.replaceAll("笔趣阁 www.biqiuge.com，最快更新一念永恒最新章节！",""));
//                    if (StringUtils.isNotEmpty(content)) {
//                        CrawlerNovel novel = new CrawlerNovel();
//                        novel.setTitle("一念永恒");
//                        novel.setChapter(chapter);
//                        novel.setContent(content);
//                        novel.setLink(link);
//                        novel.setTypeId(3);
//                        novel.setSourcesId(8);
//                        page.putField("novel", novel);
//                    }
//                }
//            }
//        }
//*/
//        /*if (!page.getUrl().regex(YNYH).match()) {
//            List<String> ynyhUrls = page.getHtml().xpath("//div[@id='list']").links().regex(YNYH).all();
//            List<String> ynyhExistLinks = novelMgrDao.getAllLinksByTypeId(3);
//            List<String> ynyhNeedAddUrls = ListUtils.getDiffrentLists(ynyhUrls,ynyhExistLinks);
//            System.out.println("一念永恒新增的urls:"+ynyhNeedAddUrls.size());
//            page.addTargetRequests(ynyhNeedAddUrls);
//        } else {
//            String chapter=page.getHtml().xpath("//div[@class='bookname']/h1/text()").toString();
//            String content=page.getHtml().xpath("//div[@id='content']/text()").toString();
//            String link = page.getUrl().toString();
//            System.out.println("chapter:::"+chapter);
//            System.out.println(content);
//            if(content != null){
//                if(StrKit.str1ContainsStr2(content,"笔趣阁 www.biqiuge.com，最快更新一念永恒最新章节！")){
//                    content = (content.replaceAll("笔趣阁 www.biqiuge.com，最快更新一念永恒最新章节！",""));
//                    System.out.println(content);
//                }
//            }
//            if (StringUtils.isNotEmpty(content)) {
//                CrawlerNovel novel = new CrawlerNovel();
//                novel.setTitle("一念永恒");
//                novel.setChapter(chapter);
//                novel.setContent(content);
//                novel.setLink(link);
//                novel.setTypeId(3);
//                novel.setSourcesId(8);
//                page.putField("novel", novel);
//            }
//        }*/
//    }
//
//
//    @Override
//    public Site getSite() {
//        return site;
//    }
//}
